﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.Hpc.Scheduler;
using System.IO;
using System.Threading;

namespace pshcli
{
    class WScheduler
    {

        public static WScheduler CurrentClusterJob = null;
        public bool IsActive = false;

        private Microsoft.Hpc.Scheduler.Scheduler ClusterSchedulerObject = null;
        private ISchedulerJob DwarfClusterJob = null;
        private Log glog;
        private GlobalSettings gsettings = null;
        private ManualResetEvent jobFinishEvt;

        private bool _plotvampir = false;
        private bool _plotjumpshot = false;

        public WScheduler(Log MyLog, GlobalSettings settings, ManualResetEvent jobFinish, bool plotVampir, bool plotJumpshot)
        {
            glog = MyLog;
            gsettings = settings;
            jobFinishEvt = jobFinish;
            _plotvampir = plotVampir;
            _plotjumpshot = plotJumpshot;
        }

        private void job_OnJobState(object sender, JobStateEventArg e)
        {
            glog.WriteLog("JOB_STATE", "Job (ID:{" + e.JobId + "}) state: {" + e.NewState + "}");

            if (e.NewState.Equals(Microsoft.Hpc.Scheduler.Properties.JobState.Finished))
            {
                DwarfBench.successfuljob = true;
                DwarfBench.waitfornext = false;
                jobFinishEvt.Set();
                return;
            }
            if (e.NewState.Equals(Microsoft.Hpc.Scheduler.Properties.JobState.Failed) ||
                e.NewState.Equals(Microsoft.Hpc.Scheduler.Properties.JobState.Canceled))
            {
                Console.WriteLine("Job ended with status: " + e.NewState);
                glog.WriteLog("JOB_STATE", "Job ended with status: " + e.NewState);
                DwarfBench.successfuljob = false;
                DwarfBench.waitfornext = false;
                jobFinishEvt.Set();
                return;
            }
        }

        public void Cancel(string Message)
        {
            ClusterSchedulerObject.CancelJob(DwarfClusterJob.Id, Message);
            IsActive = false;
        }

        /// <summary>
        /// Method blocks until cluster execution is complete.
        /// </summary>
        /// <param name="dwarf"></param>
        /// <param name="failedstring"></param>
        /// <param name="watch"></param>
        /// <returns></returns>
        public string ClusterExec(Dwarf2 dwarf, string failedstring, System.Diagnostics.Stopwatch watch)
        {
            IsActive = true;
            glog.WriteLog("EXECUTE_DWARF", "Launching scheduler execution.");
            CurrentClusterJob = this;
            try
            {
                ClusterSchedulerObject = new Scheduler();
                string ips = "";
                try
                {
                    System.Net.IPAddress[] _ips = System.Net.Dns.GetHostAddresses(gsettings.hpcscheduler);
                    foreach (System.Net.IPAddress add in _ips)
                    {
                        if (!add.ToString().Contains(':'))
                            ips = add.ToString();
                    }
                }
                catch (Exception _ips_exp)
                {
                    DwarfBench.glog.WriteLog("CLUSTER_EXEC", "Could not resolve cluster name " + gsettings.hpcscheduler);
                    DwarfBench.glog.WriteLog("CLUSTER_EXEC", _ips_exp.Message);
                    ips = gsettings.hpcscheduler; //let's try the name;
                }

                glog.WriteLog("EXECUTE_DWARF", "Cluster name " + gsettings.hpcscheduler + " was resolved as " + ips + ".");

                try
                {
                    ClusterSchedulerObject.Connect(ips);
                }
                catch (Exception connectexp)
                {
                    DwarfBench.glog.WriteLog("CLUSTER_EXEC", "Could not connect to cluster");
                    DwarfBench.glog.WriteLog("CLUSTER_EXEC", connectexp.Message);
                    failedstring = "error connecting to cluster";
                    throw new Exception("Re-throwing cluster-connect exception.");
                }

                DwarfClusterJob = ClusterSchedulerObject.CreateJob();
                DwarfClusterJob.Name =
                    "DwarfBench: " + dwarf.dwarfname + " [" +
                    dwarf.platform + "." +
                    dwarf.parallel + "." +
                    dwarf.size + "]";
                if (dwarf.mpitrace)
                    DwarfClusterJob.Name = DwarfClusterJob.Name + "[tracing on]";

                DwarfClusterJob.Project = "Dwarfs";

                ISchedulerTask executemain = DwarfClusterJob.CreateTask();

                if (!String.IsNullOrEmpty(DwarfBench._schedule_res))
                {
                    if (String.Compare("node", DwarfBench._schedule_res.ToLower()) == 0)
                    {
                        glog.WriteLog("EXECUTE_DWARF", "Scheduling to nodes.");

                        DwarfClusterJob.MaximumNumberOfNodes = gsettings.mpiproccount;
                        DwarfClusterJob.MinimumNumberOfNodes = gsettings.mpiproccount;

                        executemain.MinimumNumberOfNodes = gsettings.mpiproccount;
                        executemain.MaximumNumberOfNodes = gsettings.mpiproccount;

                    }
                    else if (String.Compare("socket", DwarfBench._schedule_res.ToLower()) == 0)
                    {
                        glog.WriteLog("EXECUTE_DWARF", "Scheduling to sockets.");
                        DwarfClusterJob.MaximumNumberOfSockets = gsettings.mpiproccount;
                        DwarfClusterJob.MinimumNumberOfSockets = gsettings.mpiproccount;

                        executemain.MinimumNumberOfSockets = gsettings.mpiproccount;
                        executemain.MaximumNumberOfSockets = gsettings.mpiproccount;

                    }
                    else
                    {
                        glog.WriteLog("EXECUTE_DWARF", "Scheduling to core.");
                        DwarfClusterJob.MaximumNumberOfCores = gsettings.mpiproccount;
                        DwarfClusterJob.MinimumNumberOfCores = gsettings.mpiproccount;

                        executemain.MinimumNumberOfCores = gsettings.mpiproccount;
                        executemain.MaximumNumberOfCores = gsettings.mpiproccount;

                    }
                }
                else
                {
                    DwarfClusterJob.MaximumNumberOfCores = gsettings.mpiproccount;
                    DwarfClusterJob.MinimumNumberOfCores = gsettings.mpiproccount;

                    executemain.MinimumNumberOfCores = gsettings.mpiproccount;
                    executemain.MaximumNumberOfCores = gsettings.mpiproccount;
                }


                if (dwarf.mpitrace)
                    DwarfClusterJob.IsExclusive = true;


                executemain.SetEnvironmentVariable("DWARF_SCRIPT_OUT", gsettings.dbclusterscriptsdir + "output\\");
                executemain.SetEnvironmentVariable("DWARF_SCRIPT_DIR", gsettings.dbclusterscriptsdir);
                executemain.SetEnvironmentVariable("DWARF_DP_FILE1", gsettings.sdeploymentdir + dwarf.dwarfname + "\\" + dwarf.GetExecName() + ".exe "); //+ gsettings.nodelocaltargetdir);
                executemain.SetEnvironmentVariable("DWARF_DP_FILE2", gsettings.sdeploymentdir + dwarf.dwarfname + "\\" + "CliTools.dll "); //+ gsettings.nodelocaltargetdir);
                executemain.SetEnvironmentVariable("DWARF_TRACE_DIR", gsettings.sdeploymentdir + dwarf.dwarfname + "\\" + "traces\\");
                executemain.SetEnvironmentVariable("DWARF_RUN_DIR", gsettings.nodelocaltargetdir + "\\");

                if (dwarf.mpitrace)
                {
                    executemain.SetEnvironmentVariable("DWARF_MPI_TRACING", "1");
                    if (_plotvampir && _plotjumpshot)
                    {
                        executemain.SetEnvironmentVariable("DWARF_MPI_CONV_TYPE", "3"); //OTF & CLOG
                    }
                    else
                    {
                        if (_plotvampir)
                        {
                            executemain.SetEnvironmentVariable("DWARF_MPI_CONV_TYPE", "1"); //OTF
                        }
                        else
                        {
                            if (_plotjumpshot)
                            {
                                executemain.SetEnvironmentVariable("DWARF_MPI_CONV_TYPE", "2"); //CLOG
                            }
                            else
                            {
                                executemain.SetEnvironmentVariable("DWARF_MPI_CONV_TYPE", "0"); //nothing
                            }
                        }
                    }
                }
                else
                {
                    executemain.SetEnvironmentVariable("DWARF_MPI_TRACING", "0");
                    executemain.SetEnvironmentVariable("DWARF_MPI_CONV_TYPE", "0"); //nothing

                }

                executemain.SetEnvironmentVariable("DWARF_EXEC_STR", dwarf.ConstructExecCommandForCluster());

                executemain.CommandLine = gsettings.dbclusterscriptsdir.Trim() + "rundwarf.bat";
                DwarfClusterJob.AddTask(executemain);

                if (dwarf.mpitrace)
                    DwarfClusterJob.IsExclusive = true;

                executemain.StdOutFilePath = gsettings.sdeploymentdir + dwarf.dwarfname + "\\" + "traces\\JobSubmissionResults.Out.txt";
                executemain.StdErrFilePath = gsettings.sdeploymentdir + dwarf.dwarfname + "\\" + "traces\\JobSubmissionResults.Err.txt";




                DwarfClusterJob.OnJobState += new EventHandler<JobStateEventArg>(job_OnJobState);

                watch.Start();

                ClusterSchedulerObject.SubmitJob(DwarfClusterJob, null, null);

                glog.WriteLog("EXECUTE_DWARF", "Cluster job started: " + DwarfClusterJob.Id + " with timeout " + gsettings.jobtimeout + "ms");


                //wait for job to complete...
                while (DwarfBench.waitfornext)
                {
                    jobFinishEvt.WaitOne(gsettings.jobtimeout);
                }
                watch.Stop();

                if (DwarfClusterJob != null)
                {
                    DwarfClusterJob.Refresh();
                    if ((DwarfClusterJob.State == Microsoft.Hpc.Scheduler.Properties.JobState.Running) ||
                        (DwarfClusterJob.State == Microsoft.Hpc.Scheduler.Properties.JobState.Validating) ||
                        (DwarfClusterJob.State == Microsoft.Hpc.Scheduler.Properties.JobState.Queued))
                    {
                        glog.WriteLog("EXECUTE_DWARF", "Timeout cancelation request sent to HPC cluster for job: " + DwarfClusterJob.Id);
                        try
                        {
                            ClusterSchedulerObject.CancelJob(DwarfClusterJob.Id, "DwarfBench-specified timeout (" + gsettings.jobtimeout + "ms) reached.");
                        }
                        catch (Exception cancelexp)
                        {
                            glog.WriteLog("EXECUTE_DWARF", "Exception occured when canceling job: " + DwarfClusterJob.Id);
                            glog.WriteLog("EXECUTE_DWARF", "Exception: " + cancelexp.Message);
                        }
                        failedstring = "time out";
                    }
                    else
                    {
                        glog.WriteLog("EXECUTE_DWARF", "Cluster job timeout caught, but job is not running. ");
                        glog.WriteLog("EXECUTE_DWARF", "Cluster job status: " + DwarfClusterJob.State);
                    }

                }



                if (dwarf.mpitrace)
                {
                    //mpi_trace_%CCP_JOBID%.%CCP_TASKID%.%CCP_TASKINSTANCEID%.etl
                    dwarf.clogf = dwarf.lmpitracedir + "mpi_trace_"
                        + executemain.TaskId + "."
                        + "0"
                        + ".etl.clog2";
                    dwarf.otff = dwarf.lmpitracedir + "mpi_trace_"
                        + executemain.TaskId + "."
                        + "0"
                        + ".etl_otf.otf";
                }

                if (DwarfBench.successfuljob)
                    dwarf.success = true;
                else
                    dwarf.success = false;
                glog.WriteLog("EXECUTE_DWARF", "Cluster execution complete for job: " + DwarfClusterJob.Id);
            }
            catch (Exception exp)
            {
                glog.WriteLog("EXECUTE_DWARF", "Cluster execution failed!");
                glog.WriteLog("EXECUTE_DWARF", "Execution error: " + exp.Message);
                glog.WriteLog("EXECUTE_DWARF", "Execution error: " + exp.Source);
                glog.WriteLog("EXECUTE_DWARF", "Execution error: " + exp.StackTrace);
                if (exp.Message.Contains("rejected"))
                    failedstring = "rejected";
                dwarf.success = false;
            }

            //reset values back for next mpi_run
            DwarfBench.waitfornext = true;
            IsActive = false;
            return failedstring;
        }
    }
}
